import scrapy


class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['dig.chouti.com']
    start_urls = ['https://dig.chouti.com/']
    index_id = 0

    def parse(self, response):
        xpath_title = response.xpath('//div[@class="link-info-con left"]')

        newtime = response.xpath('//div[25]/div//div//span//span/@data-time').extract()[0]
        print("----------------------------", newtime)

        for i in xpath_title:
            data_id = i.xpath('.//a[@class="link-title link-statistics"]/@data-id').extract_first()

            print("----------data_id-------------", data_id)
            item = {
                "type": "info",
                "index_id": self.index_id,
                "title": i.xpath('.//a[@class="link-title link-statistics"]/text()').extract_first(),
                "laiyuan": i.xpath('.//div[@class="link-from"]/text()').extract_first(),
                "zuoze": i.xpath('./div[2]//span[@class="left author-name"]/text()').extract_first(),
                "hot": i.xpath('./div[2]//span[@class="recommend-num left"]/text()').extract_first(),
                "commnum": i.xpath('.//div[2]/div[2]/a[2]/span[2]/text()').extract_first()

            }

            # 请求评论
            yield scrapy.Request(url="https://dig.chouti.com/comments/show?id=" + str(data_id),
                                 callback=self.parse_common,
                                 cb_kwargs={"index_id": self.index_id})

            self.index_id += 1

            yield item

        print("----------------------->>>>>>>>>>", newtime)
        # 请求其他数据
        yield scrapy.Request(url="https://dig.chouti.com/link/hot?afterTime=" + str(newtime), callback=self.parse_data)

    def parse_data(self, response):
        print("---数据----------")
        data_list = response.json().get("data")

        newtime = data_list[-1].get("operateTime")
        for i in data_list:
            item = {
                "type": "info",
                "index_id": self.index_id,
                "title": i.get("title"),
                "laiyuan": i.get("domain"),
                "zuoze": i.get("submittedUser").get("nick") if i.get("submittedUser") else None,
                "hot": i.get("ups"),
                "commnum": i.get("commentsCount")
            }

            self.index_id += 1
            yield item

        if self.index_id <= 100:
            # 请求其他数据
            yield scrapy.Request(url="https://dig.chouti.com/link/hot?afterTime=" + str(newtime),
                                 callback=self.parse_data)

    def parse_common(self, response, index_id):
        print("评论---------------------------------")
        data_dict = response.json().get("data")

        com_list = []

        if data_dict != {}:
            data_list = data_dict.get("commentList")
            # com_list = self.get_com_list(data_list)
            '''请求评论'''

            yield {
                "type": "com",
                "index_id": index_id,
                "com_list": com_list
            }

    # def get_com_list(self, data_list):
    #     com_list = []
    #
    #     def set_dict(data_list):
    #         com = {}
    #         for data in data_list:
    #             if data.get("childs"):
    #                 com[data.get("content")] = set_dict(data.get("childs"))
    #             else:
    #                 com[data.get("content")] = None
    #
    #             com_list.append(com)
    #
    #     set_dict(data_list)
    #
    #     return com_list
